#!/bin/sh


this=$(basename $0)
this=${this%%.*}

script=${this%%_*}.qsub


# Running directory
pwd=$(pwd)



# Clean up on all exec hosts

function clean_up {

  echo -e "\nCleaning up mdtest temp files"

  rm -f rank_file

  for h in $hosts; do 
     #echo $h
     ssh $h "rm -rf $dir/#test-dir.*"
     ssh $h "sync"
  done

  test $q -eq 0 && exit $?

}


trap clean_up SIGINT SIGQUIT SIGTERM EXIT




# Default number of nodes
num_nodes_min=1  # Overwrite with -m option
num_nodes_max=   # Overwrite with -M option


q=1  # run as batch/interactive

#
# 1. Mdtest options
#

#
# 1.1 Mdtest default params
#
z=1
b=3
I=256
N=1
i=10
#w=1048576 
w=256
dir=$pwd
cronly=
unique=
depend=

#
# 1.2 Get command line options to override 
#     the above defaults
#

#OPTERR=0
while getopts "z:b:i:I:m:M:N:w:d:W:Cuqh" option; do

    case "${option}" in
    z) 
       z=$OPTARG  
       ;;    
    b) 
       b=$OPTARG 
       ;;
    i) 
       i=$OPTARG 
       ;;    
    I) 
       I=$OPTARG  # numbner of files/dirs per task
       ;;
    m) 
       num_nodes_min=$OPTARG  # min number of nodes
       ;;
    M) 
       num_nodes_max=$OPTARG  # max number of nodes
       ;;
    N) 
       N=$OPTARG  # stat your neughbor files/dirs
       ;;
    w) 
       w=$OPTARG 
       ;;
    d)
       dir=$OPTARG
       ;;
    W) 
       depend="-W $OPTARG" 
       ;;
    C) 
       cronly=-C  # create only (no stat or remove)
       ;;   
    u) 
       unique=-u  # unique dir per task
       ;;   
    q) 
       q=0        # run interactively
       ;;   
    h) 
        mdtest -h
        exit 1
        ;;              
    esac
done


shift $(($OPTIND - 1))
#exit; 

#
# If -u flag, must set N = 0
#
if [ "$unique" = "-u" ]; then
  N=0
fi


if [ "$num_nodes_max" = "" ]; then
  num_nodes_max=$num_nodes_min
fi


mkdir -p $dir



#
# 2. Get a list of num_nodes_min ... num_nodes_max  free nodes
#


# 2.1 Number of nodes to run mdtest on
#echo "num_nodes_max = $num_nodes_max"

# 2.2 Get $num_nodes_max free nodes from PBS
nodes=$(pbsnodes -l free | sort -r | head -$num_nodes_max | awk '{print $1}')
#echo "nodes = $nodes"


# 2.3 Put the nodes on one line
hosts=$(echo $nodes | sed ':a;N;$!ba;s/\n/ /g')
#echo "hosts = $hosts"
#num_nodes_max=$(echo $hosts | wc -w) # recompute 
#echo "num_nodes computed = $num_nodes_max"


# 2.4 Create array of nodes
host_arr=($hosts)
#echo ${host_arr[0]} ${host_arr[1]} ${host_arr[2]}



#
# 3. Run mdtest on $num_nodes_min..$num_nodes_max nodes
#

reports=()

n_nodes=$num_nodes_min
inc=1


#for n_nodes in {2..4}; do
#for n_nodes in $(seq 2 2 4); do
#for n_nodes in $(seq 2 2 ${num_nodes_max}); do
#while [ $n_nodes -le 4 ]; do

while [ $n_nodes -le $num_nodes_max ]; do


  echo -e "\nRun mdtest on $n_nodes nodes with 1 process/node" 


  #
  # 3.1 Create the rank file
  #
  let n_max=n_nodes-1

  rm -f rank_file

  for idx in $(seq 0 ${n_max}); do
    #
    # Add hosts from ${host_arr[$i]}
    #
    #echo "add host ${host_arr[$i]}"
    cat >> rank_file <<EOF
rank $idx=${host_arr[$idx]} slot=1
EOF
  done




  #
  # 3.2 Extract host-list from rank file
  #
  hosts=$(cat rank_file  | awk '{print $2}' | sed 's/.*=//g' | sed ':a;N;$!ba;s/\n/ /g')
  host_list=$(echo $hosts | sed 's/ /,/g')
  #echo "hosts = $hosts"


  # 3.4 Build output file name
  p=$(printf "%02d\n" ${n_nodes})
  out_file="mdtest_p-${p}_z-${z}_b-${b}_I-${I}_N-${N}.out"
  reports=(${reports[@]} $out_file)


  # 3.5 Build command string
  command="mdtest -z $z  -b $b  -I $I  -w $w  $cronly  -N $N  $unique  -i $i  -d $dir" 

  echo "Will run $command"


  #
  # 3.6 Run mdtest
  #


  if [ $q -eq 1 ]; then 

    #
    # 3.6.1 Run mdtest as a batch job
    #

    # 3.6.1.1 Create qsub script
    cat > ${script} <<EOF
#PBS -W group_list=arcadm
##PBS -A blueridge
#PBS -N mdtest

##PBS -q large_q
#PBS -l nodes=$n_nodes:ppn=1

#PBS -k oe

cd \$PBS_O_WORKDIR

for h in \$(cat \$PBS_NODEFILE); do 
  ssh \$h "rm -rf $pwd/#test-dir.*"
  ssh \$h "sync"
done

mpirun $command > $out_file

EOF

    # 3.6.1.2 Submit job
    #cat ${this}.qsub     
    #jobID=$(qsub $depend ${script})
    submit="qsub $depend $script"
    jobID=$($submit)
    if [ $? -ne 0 ]; then 
       exit 1
    fi
    echo "  qsub ${script}  => jobID=$jobID"

    old_state="X"
    dots=0
    while [ true ]; do
      state=$(qstat -f $jobID  | grep -i job_state | awk '{print $3}')
      if [ "$state" != "C" ]; then 
          if [ "$state" != "$old_state" ]; then  
	    if [ $dots -ne 0 ]; then 
                 dots=0
                 echo " "
            fi
            echo "  $(date) wait for job $jobID to finish; current state $state"
            old_state=$state
          else 
            echo -n "."
	    let dots=dots+1
	    if [ $dots -eq 64 ]; then 
                 dots=0
                 echo ""
            fi
          fi
          sleep 6
      else
          echo ""
          break
      fi
    done

    #id=$(qselect -N mdtest -s C)
    #if [ "$jobID" != "$id" ]; then 
    #    echo "Wait for job $jobID = $id to finish"
    #    sleep 5
    #fi
 
    echo "  job $jobID = $id finished"

    # Clean up temp files
    clean_up

  else


    #
    # 3.6.2 Run mdtest interactively
    #

    # 3.6.2.2 Run
    #echo "  host_list = $host_list"
    echo "  mpirun -H $host_list $command" 
    mpirun -H $host_list -rf rank_file $command > $out_file

  fi


  # 3.7 set next node count
  if [ $n_nodes -eq 2 ]; then
    inc=2
  elif [ $n_nodes -eq 4 -o $n_nodes -eq 8 -o $n_nodes -eq 12 ]; then
    inc=4
  elif [ $n_nodes -eq 16 -o $n_nodes -eq 24 -o $n_nodes -eq 32 ]; then
    inc=8
  fi
  let n_nodes=n_nodes+inc
  

done






#  
# 4. Generate run report
#

#echo "Exit code of $this is $?"

#echo "${reports[@]}"
mdtest_report.pm ${reports[@]}


